﻿using myAC.Configuration;
using myAC.DataProviders;
using myAC.FeatureVector;
using myAC.PreProcessing.DocLevel;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace TermInsight
{
    class TFICFCalculator
    {
        static Func<myAC.DataProviders.ACInstance, SupportTopic> topicFunc = inst => (SupportTopic)(inst.OtherFeatures["Topic"]);

#if L1
        static Func<myAC.DataProviders.ACInstance, String> callCodeFunc2 = inst => topicFunc(inst).CallCodeIdAtDifferentLevels[0];
#elif L2
        static Func<myAC.DataProviders.ACInstance, String> callCodeFunc2 = inst => topicFunc(inst).CallCodeIdAtDifferentLevels[1];
#else
        static Func<myAC.DataProviders.ACInstance, String> callCodeFunc2 = inst => topicFunc(inst).CallCodeIdAtDifferentLevels[2];
#endif

        static object syncLock = new object();

        //Func<List<String> topicNameAtDifferentLevels, Int32 depth, String realFullpath>
        static Func<List<String>, Int32, String> GetRealFullpath = (topicNameAtDifferentLevels, depth) =>
        {
            StringBuilder sb = new StringBuilder();
            for (Int32 i = 0; i < depth; i++)
            {
                sb.Append(topicNameAtDifferentLevels[i]);
                sb.Append("\\");
            }
            return sb.ToString();
        };


        public static void Run()
        {
            MSSolveSRProvider2 mss = new MSSolveSRProvider2();
            String rootCallCodes = "1001856"; //"1001856,1002356,1002243";
            Console.WriteLine("Getting SR data from DB...");
            ACInstance[] instsFromAllCategories = mss.GetData(rootCallCodes);
            Console.WriteLine();
            Console.WriteLine("Obtained {0} SRs in total.", instsFromAllCategories.Length);
            if (instsFromAllCategories.Length == 0)
            {
                Console.WriteLine("Failed to get data...Quit.");
                return;
            }


            Dictionary<String, String> callCode_TopicFullpath_mapping = instsFromAllCategories.GroupBy(callCodeFunc2)
                .ToDictionary(group => group.Key,
                group =>
                {
                    String callCode = group.Key;
                    SupportTopic st = topicFunc(group.First());
                    List<String> callCodeAtDifferentLevels = st.CallCodeIdAtDifferentLevels;
                    Int32 realCallCodeLevel = callCodeAtDifferentLevels.IndexOf(callCode) + 1; // due to the extending, the first appearence of the CallCode indicates the real depth.
                    return GetRealFullpath(st.CallCodeNameAtDifferentLevels, realCallCodeLevel);
                });

            //pre-processing!! maybe only remove the stop words + space/return replacement.
            Console.WriteLine("Preprocessing text features of the surviving SRs at DOC LEVEL.");
            Int32 instsAmount = instsFromAllCategories.Length;
            Parallel.For(0, instsAmount, i =>
            {
                ACInstance inst = instsFromAllCategories[i];
                DocLevelPipeline docPipeline = new DocLevelPipeline();
                inst.TextFeatures["Processed_Ngim_ExternalTitle"] = docPipeline.Process(inst.TextFeatures["Ngim_ExternalTitle"]);
                docPipeline.Reset(); // Reset the pipeline state for next preprocessing.
                inst.TextFeatures["Processed_Ngim_CustomerReportedSymptoms"] = docPipeline.Process(inst.TextFeatures["Ngim_CustomerReportedSymptoms"]);
            });


            var allCategories = instsFromAllCategories.GroupBy(callCodeFunc2);
            Dictionary<String, Dictionary<String, Int32>> CTM = new Dictionary<string, Dictionary<string, int>>();  //Category-Term Matrix

            Char[] separators = new[] { ' ' };
            //Summarize words for each category.
            foreach (var category in allCategories)
            {
                String categoryName = category.Key;
                var instsOfThisCategory = category.ToArray();
                Dictionary<String, Int32> CTV = new Dictionary<string, int>();  //Category-Term Vector
                Parallel.ForEach(instsOfThisCategory, inst =>
                {
                    String[] titleWords = inst.TextFeatures["Processed_Ngim_ExternalTitle"].ToString().Split(separators, StringSplitOptions.RemoveEmptyEntries);
                    foreach (var word in titleWords)
                    {
                        AddWordToCTV(word, CTV);
                    }

                    String[] bodyWords = inst.TextFeatures["Processed_Ngim_CustomerReportedSymptoms"].ToString().Split(separators, StringSplitOptions.RemoveEmptyEntries);
                    foreach (var word in bodyWords)
                    {
                        AddWordToCTV(word, CTV);
                    }
                });

                CTM[categoryName] = CTV;
            }

            //Calculate TFICF(word,cateoryX,allCategorires) for each cateogry's words.

            Dictionary<String, Dictionary<String, double>> TFICF_AllCategory = new Dictionary<string, Dictionary<string, double>>();

            foreach (var category in CTM.Keys)
            {
                Dictionary<String, Int32> CTV = CTM[category];
                Dictionary<String, double> TFICF_OneCategory = new Dictionary<string, double>();  // <word, TF*ICF>
                Parallel.ForEach(CTV.Keys, word =>
                {
                    double tf = TF(word, CTV);
                    double icf = ICF(word, CTM);
                    double tficf = TFICF(tf, icf);
                    RecordTFICFValue(word, tficf, TFICF_OneCategory);
                });

                TFICF_AllCategory[category] = TFICF_OneCategory;
            }

            //sort to reveal the significant words for each category.
            Dictionary<String, List<KeyValuePair<String, double>>> sorted_TFICF_AllCategory = new Dictionary<string, List<KeyValuePair<string, double>>>();
            Parallel.ForEach(TFICF_AllCategory, kvPair =>
            {
                String category = kvPair.Key;
                var tficf_OneCategory = kvPair.Value;
                var sorted_tficf_OneCategory = tficf_OneCategory.OrderByDescending(kv => kv.Value).ToList();
                RecordedSortedTFICFValue(category, sorted_tficf_OneCategory, sorted_TFICF_AllCategory);
            });

            Int32 significanceRank = 20;   // I think we should set a threshold value instead of a ranking number.
            StringBuilder sb = new StringBuilder();
            String format = "{0}:{1}\t";

            foreach (var pair in sorted_TFICF_AllCategory)
            {
                String category = pair.Key;
                var sorted_tficf_OneCategory = pair.Value;

                sb.AppendLine(String.Format("Category:[{0}][{1}]", category, callCode_TopicFullpath_mapping[category]));
                for (Int32 i = 0; i < sorted_tficf_OneCategory.Count && i < significanceRank; i++)
                {
                    var word_tficf = sorted_tficf_OneCategory[i];
                    string word_tficf_string = string.Format(format, word_tficf.Key, word_tficf.Value);
                    sb.Append(word_tficf_string);
                }
                sb.Append("\r\n\r\n");
            }

            String filename = DateTime.Now.Ticks.ToString() + ".keywords.txt";
            File.AppendAllText(filename, sb.ToString());
        }

        private static void AddWordToCTV(String word, Dictionary<String, Int32> CTV)
        {
            lock (syncLock)
            {
                if (CTV.ContainsKey(word))
                    CTV[word]++;
                else
                    CTV[word] = 1;
            }
        }

        private static void RecordTFICFValue(String word, double tficf, Dictionary<String, double> TFICF_OneCategory)
        {
            lock (syncLock)
            {
                TFICF_OneCategory[word] = tficf;
            }
        }

        private static void RecordedSortedTFICFValue(String category, List<KeyValuePair<String, double>> sorted_TFICF_OneCategory,
            Dictionary<String, List<KeyValuePair<String, double>>> sorted_TFICF_AllCategory)
        {
            lock (syncLock)
            {
                sorted_TFICF_AllCategory[category] = sorted_TFICF_OneCategory;
            }
        }

        private static double TFICF(double tf, double icf)
        {
            return tf * icf;
        }

        private static double ICF(String word, Dictionary<String, Dictionary<String, Int32>> CTM)
        {
            double N = CTM.Count; // category total number

            double categoryContainingTheWord = CTM.Count(pair =>
            {
                var ctv = pair.Value;
                return ctv.ContainsKey(word);
            });

            return Math.Log(N / (categoryContainingTheWord + 1));
        }

        private static double TF(String word, Dictionary<String, Int32> CTV)
        {
            #region Augmented Frenquency
            //double rawFrenquency = CTV[word];
            //double maxRawFrenquency = CTV.Max(pair => pair.Value);
            //double tf = 0.5 + (0.5 * rawFrenquency) / maxRawFrenquency; //Ref: http://en.wikipedia.org/wiki/Tf%E2%80%93idf
            #endregion

            #region Log Frenquency
            //double rawFrenquency = CTV[word];
            //double tf = Math.Log(rawFrenquency + 1);
            #endregion

            #region Raw Frequency
            double tf = CTV[word];  // it seems the raw frequency is the best one.
            #endregion
            return tf;
        }
    }
}
